package indexer.processer

import us.codecraft.webmagic.Site
import us.codecraft.webmagic.selector.Selectable

class DingdianPageProcessor : UrlPageProcessor{

    override fun getUrl(): String = "http://www.x23us.com/"

    override fun getSite(): Site {
        return Site.me().setRetryTimes(3).setSleepTime(3000).setCycleRetryTimes(3).setTimeOut(180_000).setRetrySleepTime(10_000)
    }

    override fun process(page: us.codecraft.webmagic.Page) {
        val html = page.html

        val urls = html.links().regex("https?://.*\\.x23us.com.*").all()
        page.addTargetRequests(urls)

        val mainInfo: Selectable = html.xpath("//div[@class='bdsub']/dl")

        if (mainInfo.match()) {
            val name = mainInfo.xpath("//dd[1]/h1/text()").get()
            if(name != null && name.contains("最新章节更新列表")){
                val author = mainInfo.xpath("//dd[2]/h3/text()").get()
                val url = page.request.url
                val idx = name.indexOf("最新章节更新列表")
                page.putField("name",name.substring(0,idx).trim())
                page.putField("url",url)
                page.putField("author",author)
            }else{
                page.setSkip(true)
            }


        } else {
            page.setSkip(true)

        }


    }
}